package test;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

public class ProxySpider {
	private static int connectionRequestTimeout = 5000;
	private static int connectTimeout = 5000;
	private static int socketTimeout = 5000;	
	private static String baiUrl = "http://www.baidu.com";
	private static String googleUrl = "http://www.google.com";
	private static String proxyUrl = "http://www.xici.net.co/nn/";
	private static String foreignProxyUrl = "http://www.xici.net.co/wn/";
	
	public static void main(String[] args) {
		//从给定网址中获取代理
		HashMap<String,String> proxys = (HashMap<String, String>) fromHuaChiGetIP();
		//可以试用的代理
		HashMap<String,String> goodProxys = new HashMap<String, String>();
		
		//检查代理是否可用
		HttpHost httphost = null;
		for(String host : proxys.keySet()){
			String port = proxys.get(host);
			httphost = new HttpHost((String) host, Integer.parseInt(port));
			if(checkProxy(httphost,baiUrl)){
				goodProxys.put(host, port);
			}
		}
		//输出可用代理
		for(String host : goodProxys.keySet()){
			System.out.println(host+":"+goodProxys.get(host));
		}
	}

	/**
	 * 从花刺 http://www.xici.net.co/ 获取12个代理IP
	 * 
	 * @return
	 */
	private static Map<String, String> fromHuaChiGetIP() {
		HttpGet request = new HttpGet("http://www.xici.net.co/nn/");
		CloseableHttpClient httpclient = HttpClientBuilder.create().build();
		HttpResponse response = null;
		HttpEntity entity = null;
		Document dc = null;
		Map<String, String> hostMap = new HashMap<String, String>();
		String host = null;
		String port = null;

		try {
			response = httpclient.execute(request);
			if (response.getStatusLine().getStatusCode() != 200) {
				return hostMap;
			}
			entity = response.getEntity();
			dc = Jsoup.parse(EntityUtils.toString(entity));
		} catch (IOException e) {
			e.printStackTrace();
			return hostMap;
		} finally {
			try {
				httpclient.close();
			} catch (IOException e) {
			}
		}
		if (dc == null) {
			return hostMap;
		}
		for (int i = 2; i < 101; i++) {
			host = dc.select("#ip_list > tbody > tr:nth-child(" + i + ") > td:nth-child(3)").text();
			port = dc.select("#ip_list > tbody > tr:nth-child(" + i + ") > td:nth-child(4)").text();
			if (host == null || port == null || host.equals("") || port.equals("")) {
				continue;
			}
			hostMap.put(host, port);
		}
		return hostMap;
	}
	
	/**
	 * 验证代理是否有效，并且必须在connectTimeout时间内返回才合格
	 * 
	 * @param host
	 * @return
	 */
	private static boolean checkProxy(HttpHost host,String targetUrl) {
		if (host == null) {
			return false;
		}
		HttpGet request = new HttpGet(targetUrl);
		CloseableHttpClient client = HttpClientBuilder.create().setProxy(host)
				.setDefaultRequestConfig(RequestConfig.custom()//
						.setAuthenticationEnabled(true)//
						.setConnectionRequestTimeout(connectionRequestTimeout)//
						.setConnectTimeout(connectTimeout)// /
						.setSocketTimeout(socketTimeout)//
						.setRedirectsEnabled(true)//
						.setRelativeRedirectsAllowed(true)//
						.setMaxRedirects(5)//
						.build()//
				).build();
		HttpResponse response = null;
		try {
			response = client.execute(request);
			if (response.getStatusLine().getStatusCode() == 200) {
				return true;
			}
			throw new Exception();
		} catch (Exception e) {
			System.out.println(e);
			return false;
		} finally {
			if (client != null) {
				try {
					client.close();
				} catch (IOException e) {
					System.out.println("checkProxy httpclient关闭出错");
				}
			}
		}
	}
}
